import re
import sys

import facade
import parsel
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
configfile = BaseDir.get_new_path(curPath, "db.ini")


class DownHome(object):
    def __init__(self):
        self.url = "https://www.astm.org/DIGITAL_LIBRARY/JOURNALS/index.html"
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Host": "www.astm.org",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }
        self.logger = facade.get_streamlogger()
        self.mysqlutile = facade.MysqlUtiles(configfile, "db", logger=self.logger)

    def down_home(self):
        BoolResult, errString, r = facade.BaseRequest(self.url,
                                                      headers=self.headers,
                                                      timeout=(30, 60),
                                                      verify=False)
        if BoolResult:
            # print(r.text)
            Listdate = []
            selecter = parsel.Selector(text=r.text)
            # select = selecter.xpath(
            #     "//mcx_snippet_start/div/div[4]/div[1]/div[1]/div[contains(@class,'row')]")
            select = selecter.xpath(
                 '//div[@class="row landing-page clearfix"]/div[1]/div[1]/div[contains(@class,"row")]')
            for row in select:
                # print(row)
                selectspan2 = (row.xpath("./div"))
                # print(selectspan2)
                i = 0
                for span2 in selectspan2:
                    print(span2)
                    if i in (0, 2):
                        herf = span2.xpath("./a/@href").get()
                        imageherf = span2.xpath("./a/img/@src").get()
                    if i in (1, 3):
                        herf = span2.xpath("./p/b/a/@href").get()
                        source = span2.xpath("./p/b/a/text()").getall()
                        source = "".join(source)
                    if i == 1:
                        searchObj = re.search(r'\((.*?)\)', source)
                        jx = searchObj.group(1)
                        Listdate.append((source, herf, imageherf, jx))
                    if i == 3:
                        searchObj = re.search(r'\((.*?)\)', source)
                        jx = searchObj.group(1)
                        Listdate.append((source, herf, imageherf, jx))
                    i += 1
            print(len(Listdate))
            print(Listdate)
            sql = "replace into journal(`sourcename`,`url`,`imageurl`,`journal`) values (%s,%s,%s,%s)"
            self.mysqlutile.ExeSqlMany(sql, Listdate)
        else:
            print("下载错误")


def main():
    down = DownHome()
    down.down_home()


if __name__ == "__main__":
    main()
